data->done = TRUE;
}
-gboolean
-_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
- GPtrArray *mirrorlist,
- const char *filename,
- OstreeFetcherRequestFlags flags,
- GBytes **out_contents,
- guint64 max_size,
- GCancellable *cancellable,
- GError **error)
+static gboolean
+_ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fetcher,
+ GPtrArray *mirrorlist,
+ const char *filename,
+ OstreeFetcherRequestFlags flags,
+ GBytes **out_contents,
+ guint64 max_size,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
g_autoptr(GMainContext) mainctx = NULL;
return ret;
}
+gboolean
+_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
+ GPtrArray *mirrorlist,
+ const char *filename,
+ OstreeFetcherRequestFlags flags,
+ guint n_network_retries,
+ GBytes **out_contents,
+ guint64 max_size,
+ GCancellable *cancellable,
+ GError **error)
+{
+ g_autoptr(GError) local_error = NULL;
+ guint n_retries_remaining = n_network_retries;
+
+ do
+ {
+ g_clear_error (&local_error);
+ if (_ostree_fetcher_mirrored_request_to_membuf_once (fetcher, mirrorlist,
+ filename, flags,
+ out_contents, max_size,
+ cancellable, &local_error))
+ return TRUE;
+ }
+ while (_ostree_fetcher_should_retry_request (local_error, n_retries_remaining--));
+
+ g_assert (local_error != NULL);
+ g_propagate_error (error, g_steal_pointer (&local_error));
+ return FALSE;
+}
+
/* Helper for callers who just want to fetch single one-off URIs */
gboolean
_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
OstreeFetcherURI *uri,
OstreeFetcherRequestFlags flags,
+ guint n_network_retries,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
g_ptr_array_add (mirrorlist, uri); /* no transfer */
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, flags,
- out_contents, max_size,
+ n_network_retries, out_contents, max_size,
cancellable, error);
}
NULL);
#endif
}
+
+/* Check whether a particular operation should be retried. This is entirely
+ * based on how it failed (if at all) last time, and whether the operation has
+ * some retries left. The retry count is set when the operation is first
+ * created, and must be decremented by the caller. (@n_retries_remaining == 0)
+ * will always return %FALSE from this function.
+ *
+ * FIXME: In future, we may decide to use transient failures like this as a hint
+ * to prioritise other mirrors for a particular pull operation (for example). */
+gboolean
+_ostree_fetcher_should_retry_request (const GError *error,
+ guint n_retries_remaining)
+{
+ if (error == NULL)
+ g_debug ("%s: error: unset, n_retries_remaining: %u",
+ G_STRFUNC, n_retries_remaining);
+ else
+ g_debug ("%s: error: %u:%u %s, n_retries_remaining: %u",
+ G_STRFUNC, error->domain, error->code, error->message,
+ n_retries_remaining);
+
+ if (error == NULL || n_retries_remaining == 0)
+ return FALSE;
+
+ /* Return TRUE for transient errors. */
+ if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_TIMED_OUT) ||
+ g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_NOT_FOUND) ||
+ g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_UNREACHABLE) ||
+#if !GLIB_CHECK_VERSION(2, 44, 0)
+ g_error_matches (error, G_IO_ERROR, G_IO_ERROR_BROKEN_PIPE) ||
+#else
+ g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CONNECTION_CLOSED) ||
+#endif
+ g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_NOT_FOUND) ||
+ g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_TEMPORARY_FAILURE))
+ {
+ g_debug ("Should retry request (remaining: %u retries), due to transient error: %s",
+ n_retries_remaining, error->message);
+ return TRUE;
+ }
+
+ return FALSE;
+}
#define OSTREE_REPO_PULL_CONTENT_PRIORITY (OSTREE_FETCHER_DEFAULT_PRIORITY)
#define OSTREE_REPO_PULL_METADATA_PRIORITY (OSTREE_REPO_PULL_CONTENT_PRIORITY - 100)
+/* Arbitrarily chosen number of retries for all download operations when they
+ * receive a transient network error (such as a socket timeout) — see
+ * _ostree_fetcher_should_retry_request(). This is the default value for the
+ * `n-network-retries` pull option. */
+#define DEFAULT_N_NETWORK_RETRIES 5
+
typedef enum {
OSTREE_FETCHER_SECURITY_STATE_CA_PINNED,
OSTREE_FETCHER_SECURITY_STATE_TLS,
gboolean dry_run;
gboolean dry_run_emitted_progress;
gboolean legacy_transaction_resuming;
+ guint n_network_retries;
enum {
OSTREE_PULL_PHASE_FETCHING_REFS,
OSTREE_PULL_PHASE_FETCHING_OBJECTS
gboolean object_is_stored;
OstreeCollectionRef *requested_ref; /* (nullable) */
+ guint n_retries_remaining;
} FetchObjectData;
typedef struct {
char *to_revision;
guint i;
guint64 size;
+ guint n_retries_remaining;
} FetchStaticDeltaData;
typedef struct {
scan_one_metadata_object (pull_data, checksum, scan_data->objtype,
scan_data->path, scan_data->recursion_depth,
scan_data->requested_ref, pull_data->cancellable, &error);
+
+ /* No need to retry scan tasks, since they’re local. */
check_outstanding_requests_handle_error (pull_data, &error);
scan_object_queue_data_free (scan_data);
fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
+ guint n_network_retries,
char **out_contents,
GCancellable *cancellable,
GError **error)
g_autoptr(GBytes) bytes = NULL;
if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION,
+ n_network_retries,
&bytes,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
static gboolean
fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
OstreeFetcherURI *uri,
+ guint n_network_retries,
char **out_contents,
GCancellable *cancellable,
GError **error)
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
g_ptr_array_add (mirrorlist, uri); /* no transfer */
return fetch_mirrored_uri_contents_utf8_sync (fetcher, mirrorlist,
- NULL, out_contents,
+ NULL, n_network_retries,
+ out_contents,
cancellable, error);
}
pull_data->n_imported_content++;
g_assert_cmpint (pull_data->n_outstanding_content_write_requests, >, 0);
pull_data->n_outstanding_content_write_requests--;
+ /* No retries for local reads. */
check_outstanding_requests_handle_error (pull_data, &local_error);
}
if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
pull_data->meta_mirrorlist,
- filename, &ret_contents,
+ filename, pull_data->n_network_retries,
+ &ret_contents,
cancellable, error))
return FALSE;
pull_data->n_fetched_deltapart_fallbacks++;
out:
pull_data->n_outstanding_content_write_requests--;
+ /* No retries for local writes. */
check_outstanding_requests_handle_error (pull_data, &local_error);
fetch_object_data_free (fetch_data);
}
out:
pull_data->n_outstanding_content_fetches--;
- check_outstanding_requests_handle_error (pull_data, &local_error);
+
+ if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
+ enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data));
+ else
+ check_outstanding_requests_handle_error (pull_data, &local_error);
+
if (free_fetch_data)
- fetch_object_data_free (fetch_data);
+ g_clear_pointer (&fetch_data, fetch_object_data_free);
}
static void
pull_data->n_outstanding_metadata_write_requests--;
fetch_object_data_free (fetch_data);
+ /* No need to retry local write operations. */
check_outstanding_requests_handle_error (pull_data, &local_error);
}
out:
g_assert (pull_data->n_outstanding_metadata_fetches > 0);
pull_data->n_outstanding_metadata_fetches--;
- pull_data->n_fetched_metadata++;
- check_outstanding_requests_handle_error (pull_data, &local_error);
+
+ if (local_error == NULL)
+ pull_data->n_fetched_metadata++;
+
+ if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
+ enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data));
+ else
+ check_outstanding_requests_handle_error (pull_data, &local_error);
+
if (free_fetch_data)
- fetch_object_data_free (fetch_data);
+ g_clear_pointer (&fetch_data, fetch_object_data_free);
}
static void
out:
g_assert (pull_data->n_outstanding_deltapart_write_requests > 0);
pull_data->n_outstanding_deltapart_write_requests--;
+ /* No need to retry on failure to write locally. */
check_outstanding_requests_handle_error (pull_data, &local_error);
/* Always free state */
fetch_static_delta_data_free (fetch_data);
out:
g_assert (pull_data->n_outstanding_deltapart_fetches > 0);
pull_data->n_outstanding_deltapart_fetches--;
- pull_data->n_fetched_deltaparts++;
- check_outstanding_requests_handle_error (pull_data, &local_error);
+
+ if (local_error == NULL)
+ pull_data->n_fetched_deltaparts++;
+
+ if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
+ enqueue_one_static_delta_part_request_s (pull_data, g_steal_pointer (&fetch_data));
+ else
+ check_outstanding_requests_handle_error (pull_data, &local_error);
+
if (free_fetch_data)
- fetch_static_delta_data_free (fetch_data);
+ g_clear_pointer (&fetch_data, fetch_static_delta_data_free);
}
static gboolean
fetch_data->is_detached_meta = is_detached_meta;
fetch_data->object_is_stored = object_is_stored;
fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
+ fetch_data->n_retries_remaining = pull_data->n_network_retries;
- gboolean is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
-
- if (is_meta)
+ if (OSTREE_OBJECT_TYPE_IS_META (objtype))
pull_data->n_requested_metadata++;
else
pull_data->n_requested_content++;
if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
pull_data->meta_mirrorlist,
- "config", &contents,
- cancellable, error))
+ "config", pull_data->n_network_retries,
+ &contents, cancellable, error))
return FALSE;
g_autoptr(GKeyFile) ret_keyfile = g_key_file_new ();
fetch_data->is_detached_meta = FALSE;
fetch_data->object_is_stored = FALSE;
fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
+ fetch_data->n_retries_remaining = pull_data->n_network_retries;
ostree_repo_write_metadata_async (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
to_commit,
fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
fetch_data->size = size;
fetch_data->i = i;
+ fetch_data->n_retries_remaining = pull_data->n_network_retries;
if (inline_part_bytes != NULL)
{
out:
g_assert (pull_data->n_outstanding_metadata_fetches > 0);
pull_data->n_outstanding_metadata_fetches--;
- pull_data->n_fetched_metadata++;
+
+ if (local_error == NULL)
+ pull_data->n_fetched_metadata++;
+
+ /* FIXME: This should check _ostree_fetcher_should_retry_request(). */
check_outstanding_requests_handle_error (pull_data, &local_error);
g_clear_pointer (&fetch_data, fetch_delta_super_data_free);
GPtrArray *mirrorlist,
const char *filename,
gboolean is_metalink,
+ guint n_network_retries,
GBytes **out_bytes,
GCancellable *cancellable,
GError **error)
g_autoptr(OstreeMetalink) metalink =
_ostree_metalink_new (fetcher, filename,
OSTREE_MAX_METADATA_SIZE,
- mirrorlist->pdata[0]);
+ mirrorlist->pdata[0], n_network_retries);
_ostree_metalink_request_sync (metalink, NULL, out_bytes,
cancellable, &local_error);
{
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename,
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ n_network_retries,
out_bytes, OSTREE_MAX_METADATA_SIZE,
cancellable, error);
}
static gboolean
fetch_mirrorlist (OstreeFetcher *fetcher,
const char *mirrorlist_url,
+ guint n_network_retries,
GPtrArray **out_mirrorlist,
GCancellable *cancellable,
GError **error)
return FALSE;
g_autofree char *contents = NULL;
- if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, &contents,
- cancellable, error))
+ if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, n_network_retries,
+ &contents, cancellable, error))
return glnx_prefix_error (error, "While fetching mirrorlist '%s'",
mirrorlist_url);
GError *local_error = NULL;
g_autoptr(OstreeFetcherURI) config_uri = _ostree_fetcher_uri_new_subpath (mirror_uri, "config");
- if (fetch_uri_contents_utf8_sync (fetcher, config_uri, NULL,
- cancellable, &local_error))
+ if (fetch_uri_contents_utf8_sync (fetcher, config_uri, n_network_retries,
+ NULL, cancellable, &local_error))
g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
else
{
g_autoptr(GVariant) extra_headers = NULL;
g_autoptr(GPtrArray) mirrorlist = NULL;
const char *append_user_agent = NULL;
+ guint n_network_retries = DEFAULT_N_NETWORK_RETRIES;
if (options)
{
(void) g_variant_lookup (options, "override-url", "&s", &url_override);
(void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers);
(void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent);
+ (void) g_variant_lookup (options, "n-network-retries", "&u", &n_network_retries);
}
mainctx = g_main_context_new ();
g_str_has_prefix (url_string, "mirrorlist="))
{
if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="),
- &mirrorlist, cancellable, error))
+ n_network_retries, &mirrorlist, cancellable, error))
goto out;
}
else
mirrorlist,
"summary.sig",
metalink_url_string ? TRUE : FALSE,
+ n_network_retries,
out_signatures,
cancellable,
error))
mirrorlist,
"summary",
metalink_url_string ? TRUE : FALSE,
+ n_network_retries,
out_summary,
cancellable,
error))
* * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid
* * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches
* * append-user-agent (s): Additional string to append to the user agent
+ * * n-network-retries (u): Number of times to retry each download on receiving
+ * a transient network error, such as a socket timeout; default is 5, 0
+ * means return errors without retrying
*/
gboolean
ostree_repo_pull_with_options (OstreeRepo *self,
gboolean opt_gpg_verify_set = FALSE;
gboolean opt_gpg_verify_summary_set = FALSE;
gboolean opt_collection_refs_set = FALSE;
+ gboolean opt_n_network_retries_set = FALSE;
const char *main_collection_id = NULL;
const char *url_override = NULL;
gboolean inherit_transaction = FALSE;
(void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos);
(void) g_variant_lookup (options, "timestamp-check", "b", &pull_data->timestamp_check);
(void) g_variant_lookup (options, "append-user-agent", "s", &pull_data->append_user_agent);
+ opt_n_network_retries_set =
+ g_variant_lookup (options, "n-network-retries", "u", &pull_data->n_network_retries);
if (pull_data->remote_refspec_name != NULL)
pull_data->remote_name = g_strdup (pull_data->remote_refspec_name);
pull_data->main_context = g_main_context_ref_thread_default ();
pull_data->flags = flags;
+ if (!opt_n_network_retries_set)
+ pull_data->n_network_retries = DEFAULT_N_NETWORK_RETRIES;
+
pull_data->repo = self;
pull_data->progress = progress;
{
if (!fetch_mirrorlist (pull_data->fetcher,
baseurl + strlen ("mirrorlist="),
+ pull_data->n_network_retries,
&pull_data->meta_mirrorlist,
cancellable, error))
goto out;
goto out;
metalink = _ostree_metalink_new (pull_data->fetcher, "summary",
- OSTREE_MAX_METADATA_SIZE, metalink_uri);
+ OSTREE_MAX_METADATA_SIZE, metalink_uri,
+ pull_data->n_network_retries);
if (! _ostree_metalink_request_sync (metalink,
&target_uri,
{
if (!fetch_mirrorlist (pull_data->fetcher,
contenturl + strlen ("mirrorlist="),
+ pull_data->n_network_retries,
&pull_data->content_mirrorlist,
cancellable, error))
goto out;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ pull_data->n_network_retries,
&bytes_sig,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ pull_data->n_network_retries,
&bytes_summary,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
GVariant *options;
OstreeAsyncProgress *progress;
OstreeRepoFinder *default_finder_avahi;
+ guint n_network_retries;
} FindRemotesData;
static void
find_remotes_data_new (const OstreeCollectionRef * const *refs,
GVariant *options,
OstreeAsyncProgress *progress,
- OstreeRepoFinder *default_finder_avahi)
+ OstreeRepoFinder *default_finder_avahi,
+ guint n_network_retries)
{
g_autoptr(FindRemotesData) data = NULL;
data->options = (options != NULL) ? g_variant_ref (options) : NULL;
data->progress = (progress != NULL) ? g_object_ref (progress) : NULL;
data->default_finder_avahi = (default_finder_avahi != NULL) ? g_object_ref (default_finder_avahi) : NULL;
+ data->n_network_retries = n_network_retries;
return g_steal_pointer (&data);
}
* * `override-commit-ids` (`as`): Array of specific commit IDs to fetch. The nth
* commit ID applies to the nth ref, so this must be the same length as @refs, if
* provided.
+ * * `n-network-retries` (`u`): Number of times to retry each download on
+ * receiving a transient network error, such as a socket timeout; default is
+ * 5, 0 means return errors without retrying.
*
* @finders must be a non-empty %NULL-terminated array of the #OstreeRepoFinder
* instances to use, or %NULL to use the system default set of finders, which
g_autoptr(OstreeRepoFinder) finder_mount = NULL;
g_autoptr(OstreeRepoFinder) finder_avahi = NULL;
g_autofree char **override_commit_ids = NULL;
+ guint n_network_retries = DEFAULT_N_NETWORK_RETRIES;
g_return_if_fail (OSTREE_IS_REPO (self));
g_return_if_fail (is_valid_collection_ref_array (refs));
{
(void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
g_return_if_fail (override_commit_ids == NULL || g_strv_length ((gchar **) refs) == g_strv_length (override_commit_ids));
+
+ (void) g_variant_lookup (options, "n-network-retries", "u", &n_network_retries);
}
/* Set up a task for the whole operation. */
/* We need to keep a pointer to the default Avahi finder so we can stop it
* again after the operation, which happens implicitly by dropping the final
* ref. */
- data = find_remotes_data_new (refs, options, progress, finder_avahi);
+ data = find_remotes_data_new (refs, options, progress, finder_avahi, n_network_retries);
g_task_set_task_data (task, g_steal_pointer (&data), (GDestroyNotify) find_remotes_data_free);
/* Asynchronously resolve all possible remotes for the given refs. */
mirrorlist,
commit_filename,
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+ data->n_network_retries,
&commit_bytes,
0, /* no maximum size */
cancellable,
copy_option (&options_dict, &local_options_dict, "subdirs", G_VARIANT_TYPE ("as"));
copy_option (&options_dict, &local_options_dict, "update-frequency", G_VARIANT_TYPE ("u"));
copy_option (&options_dict, &local_options_dict, "append-user-agent", G_VARIANT_TYPE ("s"));
+ copy_option (&options_dict, &local_options_dict, "n-network-retries", G_VARIANT_TYPE ("u"));
local_options = g_variant_dict_end (&local_options_dict);
* - override-url (s): Fetch summary from this URL if remote specifies no metalink in options
* - http-headers (a(ss)): Additional headers to add to all HTTP requests
* - append-user-agent (s): Additional string to append to the user agent
+ * - n-network-retries (u): Number of times to retry each download on receiving
+ * a transient network error, such as a socket timeout; default is 5, 0
+ * means return errors without retrying
*
* Returns: %TRUE on success, %FALSE on failure
*/
projects / ostree.git / commitdiff